def MatchRANSAC(
image1, image2, ratio_thres, orient_agreement, scale_agreement):
"""
Read two images and their associated SIFT keypoints and descriptors.
Find matches between images based on acos distance.
Filter a subset of matches using RANSAC
Display the final matches.
HINT: See main_match.py on how to use this function.
"""
im1, keypoints1, descriptors1 = ReadKeys(image1)
im2, keypoints2, descriptors2 = ReadKeys(image2)
keypoints1 = np.stack(keypoints1, axis=0)
keypoints2 = np.stack(keypoints2, axis=0)
matched_pairs = FindBestMatchesRANSAC(
keypoints1, keypoints2,
descriptors1, descriptors2,
ratio_thres, orient_agreement, scale_agreement)
matched_pairs = [
[keypoints1[i], keypoints2[j]] for (i, j) in matched_pairs]
assert len(matched_pairs) > 0, "No match received"
im3 = DisplayMatches(im1, im2, matched_pairs)
return im3
def Match(image1, image2, ratio_thres):
"""
Read two images and their associated SIFT keypoints and descriptors.
Find matches between images based on acos distance.
Display the final matches.
HINT: See main_match.py on how to use this function.
"""
im1, keypoints1, descriptors1 = ReadKeys(image1)
im2, keypoints2, descriptors2 = ReadKeys(image2)
matched_pairs = FindBestMatches(
descriptors1, descriptors2, ratio_thres)
matched_pairs = [
[keypoints1[i], keypoints2[j]] for (i, j) in matched_pairs]
assert len(matched_pairs) > 0, "No match received"
im3 = DisplayMatches(im1, im2, matched_pairs)
return im3
def ReadKeys(image):
"""Input an image and its associated SIFT keypoints.
The argument image is the image file name (without an extension).
The image is read from the PGM format file image.pgm and the
keypoints are read from the file image.key.
ReadKeys returns the following 3 arguments:
image: the image (in PIL 'RGB' format)
keypoints: K-by-4 array, in which each row has the 4 values specifying
a keypoint (row, column, scale, orientation). The orientation
is in the range [-PI, PI] radians.
descriptors: a K-by-128 array, where each row gives a descriptor
for one of the K keypoints. The descriptor is a 1D array of 128
values with unit length.
"""
im = Image.open(image+'.pgm').convert('RGB')
keypoints = []
descriptors = []
first = True
with open(image+'.key','r') as f:
reader = csv.reader(f, delimiter=' ', quoting=csv.QUOTE_NONNUMERIC,skipinitialspace = True)
descriptor = []
for row in reader:
if len(row) == 2:
assert first, "Invalid keypoint file header."
assert row[1] == 128, "Invalid keypoint descriptor length in header (should be 128)."
count = row[0]
first = False
if len(row) == 4:
keypoints.append(np.array(row))
if len(row) == 20:
descriptor += row
if len(row) == 8:
descriptor += row
assert len(descriptor) == 128, "Keypoint descriptor length invalid (should be 128)."
#normalize the key to unit length
descriptor = np.array(descriptor)
descriptor = descriptor / math.sqrt(np.sum(np.power(descriptor,2)))
descriptors.append(descriptor)
descriptor = []
assert len(keypoints) == count, "Incorrect total number of keypoints read."
print("Number of keypoints read:", int(count))
descriptors = np.stack(descriptors, axis=0)
return [im,keypoints,descriptors]
def AppendImages(im1, im2):
"""Create a new image that appends two images side-by-side.
The arguments, im1 and im2, are PIL images of type RGB
"""
im1cols, im1rows = im1.size
im2cols, im2rows = im2.size
im3 = Image.new('RGB', (im1cols+im2cols, max(im1rows,im2rows)))
im3.paste(im1,(0,0))
im3.paste(im2,(im1cols,0))
return im3
def DisplayMatches(im1, im2, matched_pairs):
"""Display matches on a new image with the two input images placed side by side.
Arguments:
im1 1st image (in PIL 'RGB' format)
im2 2nd image (in PIL 'RGB' format)
matched_pairs list of matching keypoints, im1 to im2
Displays and returns a newly created image (in PIL 'RGB' format)
"""
im3 = AppendImages(im1,im2)
offset = im1.size[0]
draw = ImageDraw.Draw(im3)
for match in matched_pairs:
draw.line((match[0][1], match[0][0], offset+match[1][1], match[1][0]),fill="red",width=2)
im3.show()
return im3
def ReadData(fname):
"""
Given the fname, return the image, keypoints, and descriptors.
Note: the fname should be a path of the image, but with no extensions.
For example, '/my/path/ubc.png' should be '/my/path/ubc'
"""
with open(fname + '.pkl', 'rb') as f:
data = pkl.load(f)
im = Image.open(fname + '.png').convert('RGB')
keypoints = data['keypoints']
descriptors = data['descriptors']
return [im, keypoints, descriptors]
def FindBestMatchesXY(im_src_path, im_ref_path, ratio_thres):
"""
This function takes two image paths, fetch the corresponding keypoints
of the two image paths, find the best matches between keypoints
and return the keypoint correspondances in xy coordinates.
Inputs:
im_src_path: the path of the source image.
im_ref_path: the path of the image considered as the reference frame.
ratio_thres: threshold for ratio test.
Outputs:
xy_src: numpy array, (matches, 2), xy coordinates of keypoints in source.
xy_ref: numpy array, (matches, 2), xy coordinates of keypoints in ref.
"""
assert isinstance(im_src_path, str)
assert isinstance(im_ref_path, str)
assert isinstance(ratio_thres, float)
_, keypoints1, descriptors1 = ReadData(im_src_path)
_, keypoints2, descriptors2 = ReadData(im_ref_path)
matches = list(FindBestMatches(descriptors1, descriptors2, ratio_thres))
matches = [(keypoints1[i1], keypoints2[i2]) for (i1, i2) in matches]
# Extract the xy of the matches
yx_src, yx_ref = zip(*[(match[0][:2], match[1][:2]) for match in matches])
xy_src = np.array(yx_src)[:, [1, 0]] # yx to xy
xy_ref = np.array(yx_ref)[:, [1, 0]]
return xy_src, xy_ref
def PrepareData(image_list, ratio_thres):
"""
This function takes in a list of image paths of interests;
Extracts the keypoints correspondance between the reference image and all other images.
The first image on the image_list is the reference image.
Note: there is no RANSAC performed.
Inputs:
image_list: a list of paths to the images (with no extensions)
ratio_thres: the threshold for doing the ratio test of keypoint correspondance.
Outputs:
xy_src_list: numpy array, (num_matches, 2)
xy_ref_list: numpy array, (num_matches, 2)
im_list: a list of np.array, where each np.array is an image.
"""
assert isinstance(image_list, list)
assert len(image_list) > 1, "Need at leat two images to do stiching"
assert isinstance(image_list[0], str)
assert isinstance(ratio_thres, float)
assert ratio_thres >= 0.0
assert ratio_thres <= 1.0
xy_src_list = []
xy_ref_list = []
ref_image = image_list[0]
image_list = image_list[1:]
for src_image in image_list:
xy_src, xy_ref = FindBestMatchesXY(
src_image, ref_image, ratio_thres)
if xy_src.shape[0] >= 4:
xy_src_list.append(xy_src)
xy_ref_list.append(xy_ref)
im_ref, _, _ = ReadData(ref_image)
im_list = [np.array(im_ref)] + [
np.array(ReadData(img)[0]) for img in image_list]
return xy_src_list, xy_ref_list, im_list
def MergeWarppedImages(canvas_height, canvas_width, warp_list):
"""
Wrap a list of images in the reference frame into one canvas.
Note:
each image is a numpy array of shape (canvas_height, canvas_width, 3)
The first image in the warp_list is the reference image
"""
assert isinstance(canvas_height, int)
assert isinstance(canvas_width, int)
assert isinstance(warp_list, list)
canvas = np.zeros((canvas_height, canvas_width, 3), dtype=np.uint8)
im_ref = warp_list[0] # reference image in reference frame
assert im_ref.dtype == np.uint8
canvas[:im_ref.shape[0], :im_ref.shape[1]] = im_ref
alpha = 0.5
for wrap in warp_list[1:]:
assert isinstance(wrap, np.ndarray)
assert wrap.shape == canvas.shape
assert wrap.dtype == np.uint8
mask_wrap = Image.fromarray(wrap).convert('L')
mask_wrap = np.array(mask_wrap) > 0
mask_canvas = Image.fromarray(canvas).convert('L')
mask_canvas = np.array(mask_canvas) > 0
mask_intersect = np.logical_and(mask_canvas, mask_wrap)
# blend in intersected area
canvas[mask_intersect] = (
alpha*canvas[mask_intersect] +
(1-alpha)*wrap[mask_intersect]).astype(np.uint8)
canvas[mask_intersect] = (
alpha*canvas[mask_intersect] +
(1-alpha)*wrap[mask_intersect]).astype(np.uint8)
# copy in non-interected area
mask_empty = np.logical_not(mask_intersect)
canvas[mask_empty] += wrap[mask_empty]
return canvas
def ProjectImages(
xy_src_list, xy_ref_list, im_list,
canvas_height, canvas_width, num_iter, tol):
"""
This function takes in a list of images, and the points correspondance between
the reference image and other images; computes the homography from every source
image to the reference image using RANSAC; warp each source image to the reference
image frame using each homography computed.
Inputs:
xy_src_list: a list of np array, each element is keypoint correspondance
between a source image to the reference image, in xy coordinates.
xy_ref_list: a list of np array, each element is keypoint correspondance
between a source image to the reference image, in xy coordinates.
im_list: all images in np.array form, the firs element is the reference image.
canvas_height, canvas_width: the dimension of the canvas to copy the warps over.
num_iter: number of RANSAC iterations in RANSACHomography
tol: the Euclidean tolerance for keypoints matching projection.
Outputs:
A list of images in np.array form after they have been projected to
the reference frame.
"""
assert isinstance(xy_src_list, list)
assert isinstance(xy_ref_list, list)
assert isinstance(im_list, list)
assert isinstance(canvas_height, int)
assert isinstance(canvas_width, int)
assert isinstance(num_iter, int)
assert isinstance(tol, (int, float))
assert len(xy_src_list) == len(xy_ref_list)
assert len(xy_src_list) + 1 == len(im_list), \
"Num of source images + 1 == num of all images"
homo_list = []
for xy_src, xy_ref in zip(xy_src_list, xy_ref_list):
h = RANSACHomography(xy_src, xy_ref, num_iter, tol)
homo_list.append(h)
warp_list = [im_list[0]]
im_list = im_list[1:]
assert len(im_list) == len(homo_list)
for im, h in zip(im_list, homo_list):
result = cv2.warpPerspective(im, h, (canvas_width, canvas_height))
warp_list.append(result)
return warp_list
def VisualizePointProj(xy_src, xy_ref, xy_proj, im_src, im_ref):
assert isinstance(xy_src, np.ndarray)
assert isinstance(xy_ref, np.ndarray)
assert isinstance(xy_proj, np.ndarray)
assert isinstance(im_src, np.ndarray)
assert isinstance(im_ref, np.ndarray)
assert xy_src.shape == xy_ref.shape
assert xy_src.shape == xy_proj.shape
fig, axes = plt.subplots(
1, 2, figsize=(30, 30), gridspec_kw={'width_ratios': [1, 2]})
for xy_a, xy_b in zip(xy_proj, xy_ref):
x1, y1 = xy_a
x2, y2 = xy_b
axes[1].plot([x1, x2],[y1, y2], 'w-', linewidth=2)
axes[0].imshow(im_src)
axes[0].scatter(xy_src[:, 0], xy_src[:, 1], c='#fafba4', s=100, marker='.')
axes[0].title.set_text('Source Image')
axes[1].imshow(im_ref)
axes[1].scatter(xy_proj[:, 0], xy_proj[:, 1], c='#fafba4', s=100, marker='.')
axes[1].scatter(xy_ref[:, 0], xy_ref[:, 1], c='#d63447', s=100, marker='.')
axes[1].title.set_text('Reference Image')
fig.show()
input('Press any key to exit the program')
